#!/usr/bin/perl -w

BEGIN {
        if(!exists($::ENV{TRUECL_CODE})) {
                die("Missing TRUECL_CODE environment variable.");
        }
        if(!exists($::ENV{TRUECL_LOG})) {
                die("Missing TRUECL_LOG environment variable.");
        }
        if(!exists($::ENV{TRUECL_CFG})) {
                die("Missing TRUECL_CFG environment variable.");
        }
	push @::INC,"$::ENV{TRUECL_CODE}/lib";
}

#################################################################################
# This command is useful to attempt to correctly mount a single file system	#
# for an application. It is typically called from the lha_buildapp to remount	#
# new replicated file systems though could be called from the command line	#
# if necessary.									#
#################################################################################

use strict;
use Carp;
use Carp qw(cluck);
use Getopt::Long;
use cllogging;
use File::Basename;
use Sys::Hostname;
use Data::Dumper;
use lockd_client;
use cldb;
use clstatdb;
use clreq;
use Sys::Hostname;
use daemon_connect;
use clreqdxml;

$::_SPC=" " x length(basename($::0));
$::_PRG=basename($::0);
$::PKI=1;
eval "use Crypt::RSA;";
$::PKI=0 if $@;

if($::PKI==1 && exists($::ENV{TRUECL_PKI})) {
	$::PKI=$::ENV{TRUECL_PKI};
}

sub logit {
my $type=shift;
my $cline;
my $xx;

	confess("No log object defined.") if !defined($::LOGGER);
	for $cline (@_) {
		my @F=split(/\n/,$cline);
		for $xx (@F) {
			$::LOGGER->msg($type,$xx);
		}
	}
}

sub debug {
	logit("debug",@_);
}

sub msg {
	logit("msg",@_);
}

sub warning {
	logit("warn",@_);
}

sub error {
my $rc;

	if(scalar(@_)>1) {
		my $x=scalar(@_)-1;
		$rc=$_[$x];
		#########################################################
		# Treat the last as an exit code - if it is numeric.	#
		#########################################################
		if($rc =~ /^\d+$/) {
			@_=@_[0 .. $x-1];
		} else {
			undef $rc;
		}
	}
	logit("error",@_);
	exit($rc) if defined($rc);
}

sub is_true($) {
	return 0 if !defined($_[0]);
	return 1 if $_[0] =~ /^(1|yes|true|y)$/i;
	return 0;
}

$::ARG_debug=0;
$::ARG_allow_stale=0;
$::ARG_force=0;
$::ARG_silent=0;
$::ARG_quiet=0;
$::ARG_fs=undef;
$::ARG_verbose=0;
$::ARG_application=undef;
$::ARG_node=undef;
$::ARG_timeout=undef;
$::ARG_lwidth=undef;
$::ARG_nocleanup=0;
$::ARG_allow_invalid=0;

sub usage {
	print STDERR "
Usage: $::_PRG  --application A --fs /mnt
       $::_SPC [--debug|--verbose|--quiet|--silent] [--force]
       $::_SPC [--timeout N] [--nocleanup] [--allow_invalid] [--node N]

Written by Simon Edwards, (C) 2006-2008.
";
	exit(1);
}

if(!GetOptions(
	"allow_stale"			=> \$::ARG_allow_stale,
	"F|force"			=> \$::ARG_force,
	"V|verbose"			=> \$::ARG_verbose,
	"S|silent"			=> \$::ARG_silent,
	"Q|quiet"			=> \$::ARG_quiet,
	"D|debug"			=> \$::ARG_debug,
	"A|application=s"		=> \$::ARG_application,
	"N|node=s"			=> \$::ARG_node,
	"T|timeout=i"			=> \$::ARG_timeout,
	"W|lwidth=i"			=> \$::ARG_lwidth,
	"fs=s"				=> \$::ARG_fs,
	"allow_invalid"			=> \$::ARG_allow_invalid,
	"nocleanup"			=> \$::ARG_nocleanup,
	)) {
	usage;
}

if(defined($::ARG_lwidth)) {
	$::LOGGER=new cllogging("-",$::ARG_lwidth);
} else {
	$::LOGGER=new cllogging("-",80);
}

if(!defined($::LOGGER)) {
	print STDERR "Error: Unable to create logfile object.\n";
	exit(1);
}

#########################################################################
# Change the logging level based on command line arguments.		#
#########################################################################
$::LOGGER->set_debug if $::ARG_debug;
$::LOGGER->set_silent if $::ARG_silent;
$::LOGGER->set_quiet if $::ARG_quiet;
$::LOGGER->set_verbose if $::ARG_verbose;

if(!defined($::ARG_application)) {
	usage;
}

#########################################################################
# Clean-up: This is what typically gets called for lha_shutdown,	#
# though it can also be called when the startup fails.			#
# Actions available include:						#
# FS:	    Un-mount any file systems applicable to application.	#
# VG:	    Perform VG/DG deactivation on required nodes.		#
#########################################################################

sub clean_up {
my %todo=@_;

	if($::ARG_nocleanup) {
		msg("**");
		msg("** NO CLEAN UP PERFORMED! ");
		msg("**");
		exit(1);
	}
	msg("**");
	msg("** CLEAN UP UNDERWAY ");
	msg("**");
	my $timeout;
	if(defined($::APP_DETAILS{stoptimeout})) {
		$timeout=$::APP_DETAILS{stoptimeout};
	}
	if(!defined($timeout) && defined($::ARG_timeout)) {
		$timeout=$::ARG_timeout;
	}

	if(exists($todo{FS}) && $todo{FS} && defined($::REQD_RUN_NODE)) {
		msg("Attempting to un-mount application file systems.");
		my $results=$::REQD_RUN_NODE->run(cmd => ["unmount_app_fs","APPLICATION",$::ARG_application,"TIMEOUT",$timeout,"FS",$::ARG_fs]);
		my $abort=0;
		if(defined($results)) {
			for (keys(%$results)) {
				if($results->{$_} !~ /^OK/) {
					msg("$_ : $results->{$_}");
					$abort=1;
				} else {
					my @FF=split(/\s+/,$results->{$_});
					my $rc=$FF[2];
					my @RC=split(/\n/,$results->{$_});
					if($rc==0) {
						msg("Application file systems un-mounted successfully.");
					} else {
						if(exists($RC[1])) {
							msg(@RC[1..$#RC])
						}
						error("Failed to un-mount all file systems - aborting!!");
					}
				}
			}
			return if $abort;
		}
	}
	if(exists($todo{VG}) && $todo{VG} && defined($::REQD_RUN_NODE)) {
		msg("Attempting deactivation of Volume groups for '$::ARG_application'.");
		my $results=$::REQD_RUN_NODE->run(cmd => ["ss_run_before_shutdown_on_current","APPLICATION",$::ARG_application,"FS",$::ARG_fs]);
		my $abort=0;
		if(defined($results)) {
			for (keys(%$results)) {
				if($results->{$_} !~ /^OK/) {
					msg("$_ : $results->{$_}");
					$abort=1;
				}
			}
			if($abort) {
				error("Failed to deactivate volume group on '$::ARG_node'.");
				return;
			}
		}
	}
	my @redo=();
	if(exists($todo{VG}) && $todo{VG} && defined($::REQD_OTHER_NODES)) {
		msg("Attempting deactivation of remote Volume groups for '$::ARG_application'.");
		my $results=$::REQD_OTHER_NODES->run(cmd => ["ss_run_before_shutdown_on_non_current","APPLICATION",$::ARG_application,"FS",$::ARG_fs]);
		my $abort=0;
		if(defined($results)) {
			for (keys(%$results)) {
				if($results->{$_} =~ /^RETRY/) {
					push @redo,$_;
					next;
				}
				if($results->{$_} !~ /^OK/) {
					msg("$_ : $results->{$_}");
					$abort=1;
				}
			}
			if($abort) {
				error("Failed to deactivate volume group on one or more secondary nodes.");
				return;
			}
		}
	}
	msg("Storage group deactivation completed successfully.") if (exists($todo{VG}) && $todo{VG});
}

#########################################################################
# If no node is specified then default to the current node.		#
#########################################################################
$::ARG_node=hostname if !defined($::ARG_node);

#########################################################################
# First thing to do is to validate the node specified and the app	#
# are part of the cluster that this machine is part of.			#
#########################################################################

@::APP_NODES=();
{
	my $CLDB=new cldb("$::ENV{TRUECL_CFG}/config.db");
	if(!defined($CLDB)) {
		error("Unable to connect to 'config' database.","Reason: " . (cldb::error)[1] , 1);
	}

	my $res=$CLDB->get_rows("SELECT value FROM globals WHERE var = 'clustername';");
	if(!defined($res->[0]->{value})) {
		error("Incomplete cluster definition in local database [no cluster name].",1);
	}
	my $cluster=$res->[0]->{value};
	$res=$CLDB->get_rows("SELECT * FROM applications WHERE name = '$::ARG_application';");
	if(!defined($res->[0]->{nodes})) {
		error("Application '$::ARG_application' is unknown to cluster '$cluster'.",1);
	}
	my @nodes=split(/,/,$res->[0]->{nodes});
	if(!scalar(grep {$_ eq $::ARG_node} @nodes)) {
		error("Appliction '$::ARG_application' is not defined to run on node '$::ARG_node'.",1);
	}
	%::APP_DETAILS=%{$res->[0]};
	@::APP_NODES=@nodes;
	if(!defined($::ARG_timeout)) {
		$::ARG_timeout=$::APP_DETAILS{starttimeout};
		if(!defined($::ARG_timeout)) {
			warning("No configured or specified timeout for '$::ARG_application' - default to 60.");
			$::ARG_timeout=60;
		}
	}
}

#########################################################################
# Check cluster request daemons are running on both the local and the	#
# target node [if different!]						#
#########################################################################

msg("Checking for available Request Daemons - please wait.");
$::CLREQDXML=new clreqdxml("$::ENV{TRUECL_CFG}/clreqd.xml");
if(! defined($::CLREQDXML)) {
	error("Unable to load Request Daemon configuration info.", "Reason: " . clreqdxml::error(),1);
}

{
	my @R=(hostname);
	if($::ARG_node ne hostname) {
		push @R,$::ARG_node;
	}
	$::REQD=new clreq(hosts => \@R, clreqdxml => $::CLREQDXML, logger => $::LOGGER, timeout => $::ARG_timeout);
	my $results=$::REQD->run(cmd => ["echo"]);
	my @OK=(); my @DEAD=();
	for (keys(%$results)) {
		if($results->{$_} =~ /Alive!/) {
			push @OK,$_;
		} else {
			push @DEAD,$_;
		}
	}
	if(@DEAD) {
		error("The following list of nodes can not be contacted:",join(",",@DEAD),"Aborting.",1);
	}
	msg("Required Request Daemons [" . join(",",@OK) . "] running.");
}

# Attempt non-local storage activation if necessary
{

	my @H=grep {$_ ne $::ARG_node} @::APP_NODES;
	$::REQD_OTHER_NODES=new clreq(hosts => \@H, clreqdxml => $::CLREQDXML, logger => $::LOGGER, timeout => $::ARG_timeout);
	if(!defined($::REQD_OTHER_NODES)) {
		error("Unable to define request daemon - invalid application host list??",1);
	}
	my $results=$::REQD_OTHER_NODES->run(cmd => ["echo"]);
	my @OK=(); my @DEAD=();
	for (keys(%$results)) {
		if($results->{$_} =~ /Alive!/) {
			push @OK,$_;
		} else {
			push @DEAD,$_;
		}
	}
	if(@DEAD) {
		msg("The following list of nodes can not be contacted: " . join(",",@DEAD) . ".");
	}

	my $abort=0;
	msg("Attempting storage activation on other nodes - please wait...");
	$::REQD_OTHER_NODES=new clreq(hosts => [@OK], clreqdxml => $::CLREQDXML, logger => $::LOGGER, timeout => $::ARG_timeout);
	if(!defined($::REQD_OTHER_NODES)) {
		error("Unable to define request daemon - invalid application host list??",1);
	}
	push @OK,$::ARG_node;
	$::REQD_ALL_NODES=new clreq(hosts => [@OK,$::ARG_node], clreqdxml => $::CLREQDXML, logger => $::LOGGER, timeout => $::ARG_timeout);
	if(!defined($::REQD_ALL_NODES)) {
		clean_up(VG=>1);
		error("Unable to define request daemon - invalid application host list??",1);
	}

	#########################################################################
	# For all nodes that are defined as part of the application we need	#
	# the repl_map information for the application, so call that routine	#
	# now.									#
	#########################################################################
	my $rep_res=$::REQD_ALL_NODES->run(cmd => ["ss_repl_ip","APPLICATION",$::ARG_application]);
	my $errs=0;
	my @RR=();
	for my $cnode (@OK,$::ARG_node) {
		if(! exists($rep_res->{$cnode})) {
			error("ss_repl_ip did not get any results from node '$cnode'.");
			$errs++; next;
		}
		if($rep_res->{$cnode} !~ /^OK\s+/) {
			error("Invalid result from node '$cnode' for ss_repl_ip:");
			error($rep_res->{$cnode});
			$errs++; next;
		}
		my @F=split(/\s+/,$rep_res->{$cnode});
		push @RR,"$cnode:$F[1]";
	}
	if($errs) {
		exit(1);
	}
	debug("Running ss_run_before_start_on_non_current APPLICATION $::ARG_application FS $::ARG_fs REPL_MAP " . join(" ",@RR));
	$results=$::REQD_OTHER_NODES->run(cmd => ["ss_run_before_start_on_non_current","APPLICATION",$::ARG_application,"FS",$::ARG_fs,"REPL_MAP",join(" ",@RR)]);
	if(defined($results)) {
		for (keys(%$results)) {
			if($results->{$_} ne "OK") {
				$abort=1;
			}
		}
	}
	if($abort==1) {
		clean_up(VG=>1);
		exit 1;
	}
}
::msg("Available relevant nodes have performed non-current storage activation.");

#########################################################################
# Now perform the step on the target node to ready the storage for	#
# the application.							#
#########################################################################

{
	my @H=($::ARG_node);
	$::REQD_RUN_NODE=new clreq(hosts => \@H, clreqdxml => $::CLREQDXML, logger => $::LOGGER, timeout => $::ARG_timeout);
	if(!defined($::REQD_RUN_NODE)) {
		clean_up(VG=>1);
		error("Unable to define request daemon - invalid application host list??",1);
	}
	my $abort=0;
	::msg("Attempting storage activation on node '$::ARG_node' - please wait...");
	::debug("ss_run_before_start_on_current APPLICATION $::ARG_application ALLOW_STALE $::ARG_allow_stale FS $::ARG_fs FORCE $::ARG_force");
	my $results=$::REQD_RUN_NODE->run(cmd => ["ss_run_before_start_on_current","APPLICATION",$::ARG_application,"ALLOW_STALE",$::ARG_allow_stale,"FS",$::ARG_fs,"FORCE",$::ARG_force]);
	::debug("Results from ss_run_before_start_on_current APPLICATION $::ARG_application ALLOW_STALE $::ARG_allow_stale FORCE $::ARG_force:\n" . Dumper($results));
	if(defined($results)) {
		for (keys(%$results)) {
			if($results->{$_} ne "OK") {
				msg("$_ : $results->{$_}");
				$abort=1;
			}
		}
	}
	if($abort==1) {
		clean_up(VG=>1);
		exit 1;
	}

	#################################################################
	# At this point we also run the 				#
	# ss_run_after_start_on_non_current..				#
	#################################################################
	msg("Attempting final storage configuration on secondary nodes - please wait...");
	$results=$::REQD_OTHER_NODES->run(cmd => ["ss_run_after_start_on_non_current","APPLICATION",$::ARG_application,"FS",$::ARG_fs]);
	debug("Results from ss_run_after_start_on_non_current APPLICATION $::ARG_application:\n" . Dumper($results));
	if(defined($results)) {
		for (keys(%$results)) {
			if($results->{$_} ne "OK") {
				msg("$_ : $results->{$_}");
				$abort=1;
			}
		}
	}
	if($abort==1) {
		clean_up(VG=>1);
		exit 1;
	}

	#################################################################
	# Finally call ss_run_after_start_on_current			#
	#################################################################
	msg("Attempting final storage configuration on primary node - please wait...");
	$results=$::REQD_RUN_NODE->run(cmd => ["ss_run_after_start_on_current","APPLICATION",$::ARG_application,"FS",$::ARG_fs]);
	debug("Results from ss_run_after_start_on_current APPLICATION $::ARG_application FS $::ARG_fs:\n" . Dumper($results));
	if(defined($results)) {
		for (keys(%$results)) {
			if($results->{$_} ne "OK") {
				msg("$_ : $results->{$_}");
				$abort=1;
			}
		}
	}
	if($abort==1) {
		clean_up(VG=>1);
		exit 1;
	}

	#################################################################
	# Similarly attempt to mount the file systems...		#
	#################################################################

	$abort=0;
	msg("Attempting to mount file systems on '$::ARG_node' - please wait...");
	$results=$::REQD_RUN_NODE->run(cmd => ["mount_app_fs","APPLICATION",$::ARG_application,"FS",$::ARG_fs]);
	debug("Results from mount_app_fs APPLICATION $::ARG_application:\n" . Dumper($results));
	if(defined($results)) {
		for (keys(%$results)) {
			if($results->{$_} !~ /^OK/) {
				msg("$_ : $results->{$_}");
				$abort=1;
			} else {
				my @FF=split(/\s+/,$results->{$_});
				my $ok=$FF[1];
				my $failed=$FF[2];
				msg("File Systems mounted: OK=$ok, FAILED=$failed.");
				if($failed) {
					if($::ARG_force==0) {
						clean_up(FS=>1,VG=>1);
						error("Not all file systems mounted were mounted - so aborting.",1) if $::ARG_force==0;
					}
					warning("Not all file systems mounted - forced to continue.");
				}
			}
		}
	}
	exit(1) if $abort;
}

exit(0);
